import scrapy
from selenium import webdriver
from netease.items import NeteaseItem
import re


class NeteasySpider(scrapy.Spider):
    name = 'neteasy'
    # allowed_domains = ['www.163.com']
    start_urls = ['http://news.163.com/']
    models_urls = []

    def __init__(self):
        self.bro = webdriver.Chrome(executable_path='./chromedriver.exe')

    def parse(self, response):
        li_list = response.xpath('//*[@id="index2016_wrap"]/div[1]/div[2]/div[2]/div[2]/div[2]/div/ul/li')
        alist = [3]
        for index in alist:
            url = li_list[index].xpath('./a/@href').extract_first()
            self.models_urls.append(url)
        for url in self.models_urls:
            yield scrapy.Request(url, callback=self.parse_model)

    def parse_model(self, response):
        div_list = response.xpath('//div[@class="ndi_main"]/div')
        # print(div_list.extract())
        for div in div_list:
            title = div.xpath('.//h3/a/text()').extract_first()
            # print(title)
            new_detail_url = div.xpath('.//h3/a/@href').extract_first()
            # print(new_detail_url)
            item = NeteaseItem()
            item['title'] = title
            yield scrapy.Request(url=new_detail_url, callback=self.parse_detail, meta={'item': item})

    def parse_detail(self, response):
        content = response.xpath('//*[@id="endText"]/p/text()|//*[@class="content"]/p/text()').extract()
        # print(response.text)
        # print(content)
        content = [x.strip() for x in content if x.strip() != '']
        item = response.meta['item']
        item['content'] = ''.join(content)
        print(item)


        # yield item

    def closed(self, spider):
        self.bro.quit()
